package com.chance.cc.crawler.development.bootstrap.weibo.toRedis;

import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.queue.crawler.CrawlerQueueConfig;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import org.apache.commons.lang3.StringUtils;
import org.junit.Test;

import java.io.*;
import java.util.ArrayList;
import java.util.List;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRecordFilter.key;

/**
 * @Author Zhao.Hhuan
 * @Date Create in 2021/4/13 11:29
 * @Description
 **/
public class test {
    private static  List midList = new ArrayList<>();
    private static String path = "D:\\chance\\weibo\\read.txt";
    public static void main(String[] args) {
        String path = "C:\\Users\\Administrator\\Desktop\\xcar.txt";
        String path1 = "D:\\Documents\\WeChat Files\\wxid_l2bvaq20ulr822\\FileStorage\\File\\2021-05\\daily_xcar_keyword_20210520.txt";
        File readFile = new File(path);
        File readFile1 = new File(path1);
        List<String> list = new ArrayList<>();
        List<String> list1 = new ArrayList<>();
        try {
            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(readFile), "UTF-8"));
            BufferedReader bufferedReader1 = new BufferedReader(new InputStreamReader(new FileInputStream(readFile1), "UTF-8"));

            String s = null;
            while ((s = bufferedReader.readLine()) != null){
                list.add(s);
//                if(!list.contains(s)){
//                    list.add(s);
//                }else{
//                    System.out.println(s);
//                }
            }
            String s1 = null;
            while ((s1 = bufferedReader1.readLine()) != null){
                list1.add(s1);
//                if(!list.contains(s)){
//                    list.add(s);
//                }else{
//                    System.out.println(s);
//                }
            }

        } catch (Exception e) {
            e.printStackTrace();
        }


        System.out.println("开始");
        for (String s : list) {
            if(!list1.contains(s)){
                System.out.println(s);
            }
        }
        System.out.println("——————");

        for (String s : list1) {
            if(!list.contains(s)){
                System.out.println(s);
            }
        }
//        System.out.println("开始打印list");
//        for (String s : list) {
//            System.out.println(s);
//        }
//        System.out.println(list.size());
    }

    private static void readFile(){
        String url = "https://c.api.weibo.com/subscribe/update_subscribe.json";
        String site = "test";
        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domain)
                .httpUrl(url)
                .httpConfig(HttpConfig.me(domain))
                .filter(key)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .releaseTime(System.currentTimeMillis())
                .needWashed(true)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .build();

        crawlerRequestRecord.tagsCreator().bizTags().addDomain(domain);
        crawlerRequestRecord.tagsCreator().bizTags().addSite(site);

        CrawlerQueueConfig crawlerQueueConfig = new CrawlerQueueConfig();
        crawlerQueueConfig.setQueueName("weibo-dingyue-userId-next-record");
        crawlerQueueConfig.setQueueStorage(CrawlerQueueConfig.Storage.memory);
        crawlerQueueConfig.setQueueStructure(CrawlerQueueConfig.Structure.list);

        DevCrawlerController.builder()
                .triggerInfo(domain,domain,System.currentTimeMillis(),domain)
                .crawlerRequestQueue(crawlerQueueConfig)
//                .consoleResultPipeline()//控制台输出
                .consoleResultPipeline("console")
                .queueResultPipeline("result",crawlerQueueConfig)
                .requestRecord(crawlerRequestRecord)
                .build()
                .start();
    }


    private static final String domain = "weibo";

    private static final String access_token = "2.00o4_w1HrAaeYBedf38e38b8SnITmD";
    private static final String source = "1428199813";

    private static void subscribeUpdateSubscribeRecord(){
        String url = "https://c.api.weibo.com/subscribe/update_subscribe.json";
        String site = "test";
        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domain)
                .httpUrl(url)
                .httpConfig(HttpConfig.me(domain))
                .filter(key)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .releaseTime(System.currentTimeMillis())
                .needWashed(true)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .build();

        crawlerRequestRecord.tagsCreator().bizTags().addDomain(domain);
        crawlerRequestRecord.tagsCreator().bizTags().addSite(site);

        CrawlerQueueConfig crawlerQueueConfig = new CrawlerQueueConfig();
        crawlerQueueConfig.setQueueName("weibo-dingyue-userId-next-record");
        crawlerQueueConfig.setQueueStorage(CrawlerQueueConfig.Storage.memory);
        crawlerQueueConfig.setQueueStructure(CrawlerQueueConfig.Structure.list);

        DevCrawlerController.builder()
                .triggerInfo(domain,domain,System.currentTimeMillis(),domain)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain))
//                .consoleResultPipeline()//控制台输出
                .consoleResultPipeline("console")
                .queueResultPipeline("result",crawlerQueueConfig)
                .requestRecord(crawlerRequestRecord)
                .build()
                .start();
    }

    private static void testList(){
        int toIndex = 2;
        for (int i = 0; i < midList.size(); i += 2) {
            List<String> newList = midList.subList(i, i + toIndex);
            for (String s : newList) {
                System.out.println(s);
            }
            System.out.println(i+"==========="+i);
        }
    }
    @Test
    public  void  tt() throws IOException {
        String path1 = "E:\\chance-crawler-development\\crawler-dev-bootstrap\\src\\main\\java\\com\\chance\\cc\\crawler\\development\\bootstrap\\weibo\\toRedis\\weibo_urls.txt";//改成自己本地路径
        String path2 = "E:\\chance-crawler-development\\crawler-dev-bootstrap\\src\\main\\java\\com\\chance\\cc\\crawler\\development\\bootstrap\\weibo\\toRedis\\weibo_mid.txt";//改成自己本地路径
        BufferedReader bufferedReader1 = new BufferedReader(new InputStreamReader(new FileInputStream(path1), "UTF-8"));
        BufferedReader bufferedReader2 = new BufferedReader(new InputStreamReader(new FileInputStream(path2), "UTF-8"));
        String readLine1 = null;
        String readLine2 = null;
        while ((readLine1 = bufferedReader1.readLine()) != null) {
            readLine2 = bufferedReader2.readLine();
            if(StringUtils.isBlank(readLine1)){
                continue;
            }
            System.out.println(readLine1+"\t"+readLine2);

    }
}
    @Test
    public void parseUrlAndMid() throws IOException {
        String path = "E:\\chance-crawler-development\\crawler-dev-bootstrap\\src\\main\\java\\com\\chance\\cc\\crawler\\development\\bootstrap\\weibo\\weibo_urls.txt";//改成自己本地路径
        BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(path), "UTF-8"));
        String readLine = null;
        while ((readLine = bufferedReader.readLine()) != null) {
            readLine = bufferedReader.readLine();

            if(StringUtils.isBlank(readLine)){
                continue;
            }


        }

    }
    @Test
    public void test(){
        String s = "http[s]*://weibo.com/\\d+/[a-zA-Z0-9\\?\\_\\=]*";
        String url = "http://weibo.com/6429071871/Kozbitrcd";
        if (url.matches(s)){
            System.out.println("true");
        }
    }

}
